'''
scrapy爬取antispider7.scrape.center
对接账号池，代理池
'''

from scrapy import Request, Spider
import json
from scrapycompositedemo.items import BookItem

class BookSpider(Spider):
    name = "book"
    allowed_domains = ["antispider7.scrape.center"]
    base_url = "https://antispider7.scrape.center"
    max_page = 512


    def start_requests(self):
        for page in range(1, self.max_page + 1):
            url = f'{self.base_url}/api/book/?limit=1&offset={(page - 1) * 18}'
            yield Request(url, callback=self.parse_index)
    def parse_index(self, response):
        data = json.loads(response.text)
        resutls = data.get('results', [])
        for result in resutls:
            id = result.get('id')
            url = f'{self.base_url}/api/book/{id}'
            yield Request(url, callback=self.parse_detail, priority=2)

    def parse_detail(self, response):
        data = json.loads(response.text)
        item = BookItem()
        for field in item.fields:
            item[field] = data.get(field)
            yield item
